package main

import (
	"context"
	"encoding/json"
	"github.com/chromedp/chromedp"
	_ "strings"
	"time"

	articleSpider "github.com/PeterYangs/article-spider/v4"
)

func main() {
	cxt, _ := context.WithTimeout(context.Background(), 10000*time.Second)
	// f := shouyouzhijia()
	// f := shouyouzhijiaAuto()
	// f := jiaodiantiyu()
	f := chinaEduMore()

	s := articleSpider.NewSpider(f, articleSpider.Normal, cxt)
	// s := articleSpider.NewSpider(f, articleSpider.Auto, cxt)

	s.Start()
}

func chinaEduMore() articleSpider.Form {
	f := articleSpider.Form{
		Host:         "https://news.eol.cn",
		Channel:      "/yaowen/index_[PAGE].shtml",
		// 列表选择
		ListSelector: "body > div.main.container > div.left > div.listbox",
		// 列表文章链接
		HrefSelector: " div.list > div.fline > a",
		PageStart:    1,
		Length:       3,
		ListFields: map[string]articleSpider.Field{
			// body > div.main.container > div.left > div.listbox > div:nth-child(1) > div.fline > a
			"title": {Types: articleSpider.Text, Selector: " div.list > div.fline > a"},
		},
		DetailFields: map[string]articleSpider.Field{
			"title2": {Types: articleSpider.Text, Selector: "body > div.main.container > div.left > div.title"},
		},
		HttpHeader: map[string]string{
			"origin":"https://news.eol.cn",
			"referer":"https://news.eol.cn/",
			"sec-fetch-mode":"cors",
			"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
		},
	}
	return f
}

func shouyouzhijia() articleSpider.Form {
	f := articleSpider.Form{
		Host:         "https://www.shouyouzhijia.net",
		Channel:      "/xinwen_[PAGE]/",
		ListSelector: "body > div.main.newex.clearfix > div.LC_lef > div.lef_content > dl",
		HrefSelector: " dd > h3 > a",
		PageStart:    1,
		Length:       1,
		//只允许下载png和jpg两个类型的图片
		AllowImageExtension: []string{"png", "jpg"},
		ListFields: map[string]articleSpider.Field{
			"title": {Types: articleSpider.Text, Selector: " dd > h3 > a"},
		},
		/* DetailFields: map[string]articleSpider.Field{
			"title": {Types: articleSpider.Text, Selector: "body > div.Min-cent.W1200 > div.Min_L > div.Left_top > h1", ConversionFunc: func(data string, resList map[string]string) string {

				return strings.Replace(data, "[多图]", "", -1)
			}},
			"img": {Types: articleSpider.Image, Selector: "#content img:nth-child(1)", ImagePrefix: func(form *articleSpider.Form, imageName string) string {

				return "image"
			}},
			"content": {Types: articleSpider.HtmlWithImage, Selector: "#content"},
		}, */
		HttpHeader: map[string]string{
			"cookie":     "xx",
			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
		},
	}
	return f
}

// api 模式
func jiaodiantiyu() articleSpider.Form {
	f := articleSpider.Form{
		Host:      "http://www.tiyuxiu.com",
		Channel:   "/data/list_0_[PAGE].json?__t=16339338",
		PageStart: 1,
		Length:    10,
		DetailFields: map[string]articleSpider.Field{

			"title":   {Types: articleSpider.Text, Selector: "h1"},
			"content": {Types: articleSpider.HtmlWithImage, Selector: "#main-content"},
		},
		ApiConversion: func(html string, form *articleSpider.Form) []string {

			type list struct {
				Url string
			}

			var l []list

			json.Unmarshal([]byte(html), &l)

			var temp []string

			for _, l2 := range l {

				temp = append(temp, l2.Url)

			}

			return temp

		},
	}
	return f
}

// 下一页 自动化模式
func shouyouzhijiaAuto() articleSpider.Form {
	f := articleSpider.Form{

		Host: "https://www.shouyouzhijia.net",
		//自动化模式不需要设置页码，只需设置好第一页
		Channel: "/xinwen/",
		//爬三页
		Length: 3,

		ListSelector: "body > div.main.newex.clearfix > div.LC_lef > div.lef_content > dl",
		HrefSelector: "  dt > a",
		//下一页选择器
		AutoNextSelector: "body > div.main.newex.clearfix > div.LC_lef > div.pager > ul > li:nth-child(8)",
		//列表等待选择器
		AutoListWaitSelector: "body > div.main.newex.clearfix > div.LC_lef > div.lef_content > dl",
		//详情等待选择器
		AutoDetailWaitSelector: "body > div.Min-cent.W1200 > div.Min_L > div.Left_top > h1",
		DetailFields: map[string]articleSpider.Field{
			"title": {Types: articleSpider.Text, Selector: "body > div.Min-cent.W1200 > div.Min_L > div.Left_top > h1"},
		},
	}
	return f
}

// 加载更多
func more() articleSpider.Form {
	f := articleSpider.Form{
		Host:         "https://www.btcfans.com",
		Channel:      "/zh-cn/wallet",
		ListSelector: "body > div.page-width.page-content > div.main-content > div > div.module-content > ul > li",
		HrefSelector: " a",
		//下一页选择器
		AutoNextSelector: "body > div.page-width.page-content > div.main-content > div > div.module-content > a",
		//列表等待选择器
		AutoListWaitSelector: "body > div.page-width.page-content > div.main-content > div > div.module-content > ul > li:nth-child(1)",
		//详情等待选择器
		AutoDetailWaitSelector: "body > div.page-width.page-content > div.main-content > div.wallet-detail-page > div.info_1 > div.name > div.name-ch",
		Length:                 4,
		DetailFields: map[string]articleSpider.Field{
			"title": {ExcelHeader: "G", Types: articleSpider.Text, Selector: "body > div.page-width.page-content > div.main-content > div.wallet-detail-page > div.info_1 > div.name > div.name-ch"},
			"content": {Types: articleSpider.HtmlWithImage, Selector: "body > div.page-width.page-content > div.main-content > div.wallet-detail-page > div.wallet-des > div > p", ExcelHeader: "E", ImagePrefix: func(form *articleSpider.Form, imageName string) string {

				return "/api/uploads"
			}, ImageDir: "game[date:md]/[random:1-100]"},
			"desc":    {Types: articleSpider.Attr, Selector: "meta[name=\"description\"]", AttrKey: "content", ExcelHeader: "H"},
			"keyword": {Types: articleSpider.Attr, Selector: "meta[name=\"keywords\"]", AttrKey: "content", ExcelHeader: "K"},
			"img":     {Types: articleSpider.Image, Selector: "body > div.page-width.page-content > div.main-content > div.wallet-detail-page > div.info_1 > div.cover > img", ExcelHeader: "F", ImageDir: "game[date:md]/[random:1-100]"},
			"type":    {Types: articleSpider.Fixed, Selector: "2", ExcelHeader: "L"},
			//"size":    {Types: fileTypes.SingleField, Selector: "#dinfo > p.base > i:nth-child(3)", ExcelHeader: "M"},
		},

		//cookie
		HttpHeader: map[string]string{
			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
			"cookie":     "lang=zh-CN; lang=zh-CN; lang=zh-CN; _ga=GA1.1.1532009431.1641283813; UM_distinctid=17e24238a22739-0fc0995e9cfdad-c343365-1fa400-17e24238a2352e; guid=cff3a072d6ca30b80ee729f0884a8596f65d9a28; CNZZDATA5291371=cnzz_eid%3D1358048227-1641278212-%26ntime%3D1641338428; CNZZDATA1278599438=848177868-1641279863-%7C1641340242; Hm_lvt_ddaa34551214df42d1e5f11974f9f744=1641283822,1641346329; _csrf=3f62bc78510faa5fecfbf404cbee0ec56d1c4f3a; s_a=1; _ga_76F07DJEB4=GS1.1.1641346328.3.1.1641346978.0; Hm_lpvt_ddaa34551214df42d1e5f11974f9f744=1641346980",
		},
		//下一页模式
		AutoNextPageMode:  articleSpider.LoadMore,
		CustomExcelHeader: true,
		//爬取前置事件
		AutoPrefixEvent: func(chromedpCtx context.Context) {

			//关闭弹窗
			chromedp.Run(
				chromedpCtx,

				chromedp.Click("#Alert > div > div.sure_btn", chromedp.ByQuery),
			)

		},
	}
	return f
}
